install.packages('corpcor')
install.packages('mctest')
require(mgcv)
require(corpcor)
require(mctest)



#Importing datasets
all_sellerdata  = read.csv(file="C:\\Users\\f00456n\\Documents\\Amazon Price Dynamics\\Data Scraped\\New Data\\Modeling data sets\\Deep Fryer\\finalnumsellers_modeldata_deepfryer.csv", header=TRUE, sep=",")

modeldata = all_sellerdata[4:138,]

modeldata[is.na(modeldata)] = 0

attach(modeldata)

#Y_clust1_csnart = primary3p_Csn
#Y_clust1_csnart_lag1  = primary3p_CSNlag1
#Y_clust1_tfal = primary3p_tfal

trainY = cbind(Y_amzn_tfal,Y_amzn_hbeach,Y_amzn_csnart,Y_amzn_presto,Y_clust1_csnart,Y_clust3_csnart,Y_clust4_secura,Y_clust5_tfal)

#AMAZON TFAL
data_amzn_tfal = data.frame(cbind(Y_amzn_tfal_lag1,Y_amzn_tfal_lag2,Y_amzn_tfal_lag3,   
                                  Y_clust4_secura_lag1,
                                  min_cumsalefreq_clust5_lag1,
                                  num_csnartclust3_lag1,
                                  meannumsellerate_clust3_lag1,
                                  Y_amzn_csnart_lag1,
                                  buyboxprice_csnart_lag1,
                                  buyboxprice_tfal_lag1,
                                  mean_uniq_ASIN_clust1_lag1,
                                  maxpercsellerate_clust2_lag1,
                                  ep_topbrandlag1	,
                                  presto_topbrandlag1	,
                                  hbeach_topbrandlag1	,
                                  csnart_topbrandlag1	,
                                  secura_topbrandlag1	,
                                  tfal_topbrandlag1	,
                                  ep_bottombrandlag1	,
                                  presto_bottombrandlag1	,
                                  hbeach_bottombrandlag1	,
                                  csnart_bottombrandlag1	,
                                  secura_bottombrandlag1	,
                                  tfal_bottombrandlag1	,
                                  decr10per_3p_tfal	,
                                  decr10per_amzn_hbeach	,
                                  decr10per_amzn_tfal	,
                                  decr10per_clust3_csnart	,
                                  decr20per_3p_sec	,
                                  decr20per_amzn_csnart	,
                                  decr20per_amzn_presto	,
                                  decr20per_clust1_csnart	,
                                  decr20per_clust5_tfal	,
                                  decr5per_3p_tfal	,
                                  decr5per_amzn_hbeach	,
                                  decr5per_amzn_tfal	,
                                  decr5per_clust3_csnart	,
                                  incr10per_3p_sec	,
                                  incr10per_amzn_csnart	,
                                  incr10per_amzn_presto	,
                                  incr10per_clust1_csnart	,
                                  incr10per_clust5_tfal	,
                                  incr20per_3p_tfal	,
                                  incr20per_amzn_hbeach	,
                                  incr20per_amzn_tfal	,
                                  incr20per_clust3_csnart	,
                                  incr5per_3p_sec	,
                                  incr5per_amzn_csnart	,
                                  incr5per_amzn_presto	,
                                  incr5per_clust1_csnart	,
                                  incr5per_clust5_tfal	,
                                  decr10per_3p_sec	,
                                  decr10per_amzn_csnart	,
                                  decr10per_amzn_presto	,
                                  decr10per_clust1_csnart	,
                                  decr10per_clust5_tfal	,
                                  decr20per_3p_tfal	,
                                  decr20per_amzn_hbeach	,
                                  decr20per_amzn_tfal	,
                                  decr20per_clust3_csnart	,
                                  decr5per_3p_sec	,
                                  decr5per_amzn_csnart	,
                                  decr5per_amzn_presto	,
                                  decr5per_clust1_csnart	,
                                  decr5per_clust5_tfal	,
                                  incr10per_3p_tfal	,
                                  incr10per_amzn_hbeach	,
                                  incr10per_amzn_tfal	,
                                  incr10per_clust3_csnart	,
                                  incr20per_3p_sec	,
                                  incr20per_amzn_csnart	,
                                  incr20per_amzn_presto	,
                                  incr20per_clust1_csnart	,
                                  incr20per_clust5_tfal	,
                                  incr5per_3p_tfal	,
                                  incr5per_amzn_hbeach	,
                                  incr5per_amzn_tfal	,
                                  incr5per_clust3_csnart	,
                                  weekend	,
                                  #additional variables on product reviews
                                  max_Product_reviews_presto_lag1,
                                  Product_reviews_csnart_lag1,
                                  Product_reviews_Hbeach_lag1,
                                  Product_reviews_secura_lag1,
                                  Product_reviews_tfal_lag1,
                                  meanprodstar_presto_lag1,
                                  productstar_csnart_lag1,
                                  productstar_EP_lag1,      
                                  productstar_Hbeach_lag1,
                                  productstar_secura_lag1,
                                  productstar_tfal_lag1,
                                  #additional FEs after 2nd RnR
                                  meansellerstar_clust1_lag1,
                                  meansellerstar_clust2_lag1,
                                  meansellerstar_clust3_lag1,  
                                  meansellerstar_clust4_lag1,
                                  meansellerstar_clust5_lag1, 
                                  mean_starrateTfal_3p_lag1,
                                  mean_primeTfal_3p_lag1,
                                  seasonal_sale,
                                  ###broken down external site price changes with lags
                                  maxpricechange_HD,
                                  maxpricechange_HDlag1,
                                  maxpricechange_HDlag2,
                                  maxpricechange_HDlag3,
                                  maxpricechange_sears,
                                  maxpricechange_searslag1,
                                  maxpricechange_searslag2,
                                  maxpricechange_searslag3,
                                  maxpricechange_wm,
                                  maxpricechange_wmlag1,
                                  maxpricechange_wmlag2,
                                  maxpricechange_wmlag3))


#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS

gam10_amzn_tfal = gam(Y_amzn_tfal ~  
                        #Y_amzn_tfal_lag1	+
                       # Y_amzn_tfal_lag2	+
                        s(Y_clust4_secura_lag1)+
                        #buyboxprice_csnart_lag1 +
                        #+ mean_uniq_ASIN_clust1_lag1
                        #+maxpercsellerate_clust2_lag1+
                        buyboxprice_tfal_lag1+
                        #ep_topbrandlag1	+
                        #presto_topbrandlag1	+
                        #hbeach_topbrandlag1	+
                        #csnart_topbrandlag1	+
                        #secura_topbrandlag1	+
                        #tfal_topbrandlag1	+
                        ep_bottombrandlag1	+
                        presto_bottombrandlag1	+
                        #hbeach_bottombrandlag1	+
                       # csnart_bottombrandlag1	+
                       # secura_bottombrandlag1	+
                        #tfal_bottombrandlag1	+
                       # decr10per_3p_tfal	+
                        #decr10per_amzn_hbeach	+
                        #decr10per_amzn_tfal	+
                        decr10per_clust3_csnart	+
                       # decr20per_amzn_csnart	+
                        #decr20per_clust1_csnart	+
                        #decr5per_3p_tfal	+
                       # decr5per_amzn_hbeach	+
                        #decr5per_amzn_tfal	+
                        #decr5per_clust3_csnart	+
                        #incr10per_amzn_csnart	+
                        #incr10per_amzn_presto	+
                        #incr10per_clust1_csnart	+
                        incr20per_3p_tfal	+
                        #incr20per_amzn_hbeach	+
                        incr20per_amzn_tfal	+
                        #incr5per_amzn_csnart	+
                        #incr5per_amzn_presto	+
                      #  incr5per_clust1_csnart	+
                        #decr10per_amzn_csnart	+
                        #decr10per_amzn_presto	+
                      #  decr10per_clust1_csnart	+
                        #decr20per_amzn_hbeach	+
                        #decr20per_amzn_tfal	+
                        #decr5per_amzn_csnart	+
                        #decr5per_amzn_presto	+
                        #decr5per_clust1_csnart	+
                       # incr10per_3p_tfal	+
                        #incr10per_amzn_hbeach	+
                        #incr10per_amzn_tfal	+
                        #incr20per_amzn_csnart	+
                        #incr20per_clust1_csnart	+
                        #incr5per_3p_tfal	+
                        #incr5per_amzn_hbeach	+
                       # incr5per_amzn_tfal	+
                       # incr5per_clust3_csnart	+
                        weekend	+
                        #additional variables on product reviews
                        max_Product_reviews_presto_lag1+
                         Product_reviews_csnart_lag1+
                     # Product_reviews_Hbeach_lag1+
                     # Product_reviews_secura_lag1+
                      Product_reviews_tfal_lag1+
                     # meanprodstar_presto_lag1+
                      productstar_csnart_lag1+
                     # productstar_EP_lag1+      
                     # productstar_Hbeach_lag1+
                      productstar_secura_lag1+
                    #  productstar_tfal_lag1+
                      #additional FEs after 2nd RnR
                   # mean_starrateTfal_clust1_lag1+
                  #  mean_starrateTfal_clust2_lag1+
                  #  mean_starrateTfal_clust3_lag1+
                    #mean_starrateTfal_clust4_lag1+
                    mean_starrateTfal_3p_lag1+
                    seasonal_sale	+
                    
                    ###4th round additions
                  #  maxpricechange_HD +
                  #  maxpricechange_HDlag3 +
                   # maxpricechange_sears +
                  #  maxpricechange_searslag3 +
                    maxpricechange_wm  +
                    maxpricechange_wmlag3
                    ,family= gaussian(link ='identity') ,data=data_amzn_tfal,method="REML",optimizer=c("outer","newton"), fit = TRUE)


summary(gam10_amzn_tfal)

#AMAZON hbeach

data_amzn_hbeach = data.frame(cbind(Y_amzn_hbeach_lag1,Y_amzn_hbeach_lag2,Y_amzn_hbeach_lag3,Y_clust4_secura_lag1,
                                    min_cumsalefreq_clust5_lag1,
                                    num_csnartclust3_lag1,
                                    meannumsellerate_clust3_lag1,
                                    Y_amzn_csnart_lag1,
                                    buyboxprice_csnart_lag1,
                                    buyboxprice_Hbeach_lag1,
                                    mean_uniq_ASIN_clust1_lag1,
                                    maxpercsellerate_clust2_lag1,
                                    Y_amzn_tfal_lag1,
                                    ep_topbrandlag1	,
                                    presto_topbrandlag1	,
                                    hbeach_topbrandlag1	,
                                    csnart_topbrandlag1	,
                                    secura_topbrandlag1	,
                                    tfal_topbrandlag1	,
                                    ep_bottombrandlag1	,
                                    presto_bottombrandlag1	,
                                    hbeach_bottombrandlag1	,
                                    csnart_bottombrandlag1	,
                                    secura_bottombrandlag1	,
                                    tfal_bottombrandlag1	,
                                    decr10per_3p_tfal	,
                                    decr10per_amzn_hbeach	,
                                    decr10per_amzn_tfal	,
                                    decr10per_clust3_csnart	,
                                    decr20per_3p_sec	,
                                    decr20per_amzn_csnart	,
                                    decr20per_amzn_presto	,
                                    decr20per_clust1_csnart	,
                                    decr20per_clust5_tfal	,
                                    decr5per_3p_tfal	,
                                    decr5per_amzn_hbeach	,
                                    decr5per_amzn_tfal	,
                                    decr5per_clust3_csnart	,
                                    incr10per_3p_sec	,
                                    incr10per_amzn_csnart	,
                                    incr10per_amzn_presto	,
                                    incr10per_clust1_csnart	,
                                    incr10per_clust5_tfal	,
                                    incr20per_3p_tfal	,
                                    incr20per_amzn_hbeach	,
                                    incr20per_amzn_tfal	,
                                    incr20per_clust3_csnart	,
                                    incr5per_3p_sec	,
                                    incr5per_amzn_csnart	,
                                    incr5per_amzn_presto	,
                                    incr5per_clust1_csnart	,
                                    incr5per_clust5_tfal	,
                                    decr10per_3p_sec	,
                                    decr10per_amzn_csnart	,
                                    decr10per_amzn_presto	,
                                    decr10per_clust1_csnart	,
                                    decr10per_clust5_tfal	,
                                    decr20per_3p_tfal	,
                                    decr20per_amzn_hbeach	,
                                    decr20per_amzn_tfal	,
                                    decr20per_clust3_csnart	,
                                    decr5per_3p_sec	,
                                    decr5per_amzn_csnart	,
                                    decr5per_amzn_presto	,
                                    decr5per_clust1_csnart	,
                                    decr5per_clust5_tfal	,
                                    incr10per_3p_tfal	,
                                    incr10per_amzn_hbeach	,
                                    incr10per_amzn_tfal	,
                                    incr10per_clust3_csnart	,
                                    incr20per_3p_sec	,
                                    incr20per_amzn_csnart	,
                                    incr20per_amzn_presto	,
                                    incr20per_clust1_csnart	,
                                    incr20per_clust5_tfal	,
                                    incr5per_3p_tfal	,
                                    incr5per_amzn_hbeach	,
                                    incr5per_amzn_tfal	,
                                    incr5per_clust3_csnart	,
                                    weekend	,
                                    #additional variables on product reviews
                                    meansellerstar_clust1_lag1,     
                                    meansellerstar_clust2_lag1,
                                    meansellerstar_clust3_lag1,
                                    meansellerstar_clust4_lag1,     
                                    meansellerstar_clust5_lag1, 
                                    max_Product_reviews_presto_lag1,
                                    Product_reviews_csnart_lag1,
                                    Product_reviews_Hbeach_lag1,
                                    Product_reviews_secura_lag1,
                                    Product_reviews_tfal_lag1,
                                    meanprodstar_presto_lag1,
                                    productstar_csnart_lag1,
                                    productstar_EP_lag1,      
                                    productstar_Hbeach_lag1,
                                    productstar_secura_lag1,
                                    productstar_tfal_lag1,
                                    #additional FEs after 2nd RnR
                                    seasonal_sale,
                                    ###broken down external site price changes with lags
                                    maxpricechange_HD,
                                    maxpricechange_HDlag1,
                                    maxpricechange_HDlag2,
                                    maxpricechange_HDlag3,
                                    maxpricechange_sears,
                                    maxpricechange_searslag1,
                                    maxpricechange_searslag2,
                                    maxpricechange_searslag3,
                                    maxpricechange_wm,
                                    maxpricechange_wmlag1,
                                    maxpricechange_wmlag2,
                                    maxpricechange_wmlag3))


#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS
gam8_amzn_hbeach = gam(Y_amzn_hbeach ~ Y_amzn_csnart_lag1 + Y_clust4_secura_lag1 + 
                         buyboxprice_csnart_lag1 + s(mean_uniq_ASIN_clust1_lag1) + 
                         Y_amzn_tfal_lag1 + tfal_topbrandlag1 + decr10per_amzn_hbeach + 
                         decr20per_amzn_csnart + decr5per_amzn_hbeach + decr20per_amzn_hbeach + buyboxprice_Hbeach_lag1 + max_Product_reviews_presto_lag1 + 
                         Product_reviews_Hbeach_lag1 + 
                         Product_reviews_tfal_lag1 + 
                         meanprodstar_presto_lag1 + productstar_Hbeach_lag1 + productstar_tfal_lag1 + 
                         seasonal_sale + weekend +
                         ###4th round additions
                         #maxpricechange_HD +
                         maxpricechange_HDlag2 +
                         #maxpricechange_sears +
                         maxpricechange_searslag2 +
                         #maxpricechange_wm  +
                         maxpricechange_wmlag2
                      ,family= gaussian(link ='identity') ,data=data_amzn_hbeach ,method="REML",optimizer=c("outer","newton"), fit = TRUE)
summary(gam8_amzn_hbeach)

#AMAZON CUISINART


data_amzn_csnart = data.frame(cbind(Y_amzn_csnart_lag1,Y_amzn_csnart_lag2,Y_amzn_csnart_lag3,   
                                    Y_clust4_secura_lag1,
                                    min_cumsalefreq_clust5_lag1,
                                    num_csnartclust3_lag1,
                                    meannumsellerate_clust3_lag1,
                                    buyboxprice_csnart_lag1,
                                    mean_uniq_ASIN_clust1_lag1,
                                    maxpercsellerate_clust2_lag1,
                                    Y_amzn_tfal_lag1,
                                    ep_topbrandlag1	,
                                    presto_topbrandlag1	,
                                    hbeach_topbrandlag1	,
                                    csnart_topbrandlag1	,
                                    secura_topbrandlag1	,
                                    tfal_topbrandlag1	,
                                    ep_bottombrandlag1	,
                                    presto_bottombrandlag1	,
                                    hbeach_bottombrandlag1	,
                                    csnart_bottombrandlag1	,
                                    secura_bottombrandlag1	,
                                    tfal_bottombrandlag1	,
                                    decr10per_3p_tfal	,
                                    decr10per_amzn_hbeach	,
                                    decr10per_amzn_tfal	,
                                    decr10per_clust3_csnart	,
                                    decr20per_3p_sec	,
                                    decr20per_amzn_csnart	,
                                    decr20per_amzn_presto	,
                                    decr20per_clust1_csnart	,
                                    decr20per_clust5_tfal	,
                                    decr5per_3p_tfal	,
                                    decr5per_amzn_hbeach	,
                                    decr5per_amzn_tfal	,
                                    decr5per_clust3_csnart	,
                                    incr10per_3p_sec	,
                                    incr10per_amzn_csnart	,
                                    incr10per_amzn_presto	,
                                    incr10per_clust1_csnart	,
                                    incr10per_clust5_tfal	,
                                    incr20per_3p_tfal	,
                                    incr20per_amzn_hbeach	,
                                    incr20per_amzn_tfal	,
                                    incr20per_clust3_csnart	,
                                    incr5per_3p_sec	,
                                    incr5per_amzn_csnart	,
                                    incr5per_amzn_presto	,
                                    incr5per_clust1_csnart	,
                                    incr5per_clust5_tfal	,
                                    decr10per_3p_sec	,
                                    decr10per_amzn_csnart	,
                                    decr10per_amzn_presto	,
                                    decr10per_clust1_csnart	,
                                    decr10per_clust5_tfal	,
                                    decr20per_3p_tfal	,
                                    decr20per_amzn_hbeach	,
                                    decr20per_amzn_tfal	,
                                    decr20per_clust3_csnart	,
                                    decr5per_3p_sec	,
                                    decr5per_amzn_csnart	,
                                    decr5per_amzn_presto	,
                                    decr5per_clust1_csnart	,
                                    decr5per_clust5_tfal	,
                                    incr10per_3p_tfal	,
                                    incr10per_amzn_hbeach	,
                                    incr10per_amzn_tfal	,
                                    incr10per_clust3_csnart	,
                                    incr20per_3p_sec	,
                                    incr20per_amzn_csnart	,
                                    incr20per_amzn_presto	,
                                    incr20per_clust1_csnart	,
                                    incr20per_clust5_tfal	,
                                    incr5per_3p_tfal	,
                                    incr5per_amzn_hbeach	,
                                    incr5per_amzn_tfal	,
                                    incr5per_clust3_csnart	,
                                    weekend	,
                                    #additional variables on product reviews
                                    max_Product_reviews_presto_lag1,
                                    Product_reviews_csnart_lag1,
                                    Product_reviews_Hbeach_lag1,
                                    Product_reviews_secura_lag1,
                                    Product_reviews_tfal_lag1,
                                    meanprodstar_presto_lag1,
                                    productstar_csnart_lag1,
                                    productstar_EP_lag1,      
                                    productstar_Hbeach_lag1,
                                    productstar_secura_lag1,
                                    productstar_tfal_lag1,
                                    #additional FEs after 2nd RnR
                                    meansellerstar_clust1_lag1,     
                                    meansellerstar_clust2_lag1,
                                    meansellerstar_clust3_lag1,
                                    meansellerstar_clust4_lag1,     
                                    meansellerstar_clust5_lag1, 
                                    mean_starrateCSN_3p_lag1,
                                    mean_primeCSN_3p_lag1,
                                    seasonal_sale,
                                    ###broken down external site price changes with lags
                                    maxpricechange_HD,
                                    maxpricechange_HDlag1,
                                    maxpricechange_HDlag2,
                                    maxpricechange_HDlag3,
                                    maxpricechange_sears,
                                    maxpricechange_searslag1,
                                    maxpricechange_searslag2,
                                    maxpricechange_searslag3,
                                    maxpricechange_wm,
                                    maxpricechange_wmlag1,
                                    maxpricechange_wmlag2,
                                    maxpricechange_wmlag3))

#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS

gam7_amzn_csnart  = gam(Y_amzn_csnart  ~  
                          #Y_amzn_csnart_lag1	+
                          meannumsellerate_clust3_lag1+
                          mean_uniq_ASIN_clust1_lag1 +
                         # Y_amzn_tfal_lag1 +
                          buyboxprice_csnart_lag1+
                          #ep_topbrandlag1	+
                          #presto_topbrandlag1	+
                          #hbeach_topbrandlag1	+
                          #csnart_topbrandlag1	+
                          #secura_topbrandlag1	+
                          tfal_topbrandlag1	+
                          #ep_bottombrandlag1	+
                          #presto_bottombrandlag1	+
                          #hbeach_bottombrandlag1	+
                          # csnart_bottombrandlag1	+
                          # secura_bottombrandlag1	+
                          #tfal_bottombrandlag1	+
                        # decr10per_3p_tfal	+
                        #decr10per_amzn_hbeach	+
                          #decr10per_amzn_tfal	+
                          #decr10per_clust3_csnart	+
                         # decr20per_amzn_csnart	+
                         # decr20per_clust1_csnart	+
                          #decr5per_3p_tfal	+
                         # decr5per_amzn_hbeach	+
                          #decr5per_amzn_tfal	+
                        #  decr5per_clust3_csnart	+
                          #incr10per_amzn_csnart	+
                          #incr10per_amzn_presto	+
                          #incr10per_clust1_csnart	+
                          #incr20per_3p_tfal	+
                          #incr20per_amzn_hbeach	+
                          incr20per_amzn_tfal	+
                          #incr5per_amzn_csnart	+
                          #incr5per_amzn_presto	+
                        #  incr5per_clust1_csnart	+
                          #decr10per_amzn_csnart	+
                          #decr10per_amzn_presto	+
                          #decr10per_clust1_csnart	+
                          incr10per_clust3_csnart+
                          #decr20per_amzn_hbeach	+
                          #decr20per_amzn_tfal	+
                          #decr5per_amzn_csnart	+
                          #decr5per_amzn_presto	+
                          #decr5per_clust1_csnart	+
                          #incr10per_3p_tfal	+
                          # incr10per_amzn_hbeach	+
                         # incr10per_amzn_tfal	+
                          #incr20per_amzn_csnart	+
                          #incr20per_amzn_presto	+
                          incr20per_clust1_csnart	+
                          #incr5per_3p_tfal	+
                          #incr5per_amzn_hbeach	+
                          #incr5per_amzn_tfal	+
                          incr5per_clust3_csnart	+
                          #additional variables on product reviews
                          max_Product_reviews_presto_lag1+
                          Product_reviews_csnart_lag1+
                          meanprodstar_presto_lag1+
                          productstar_csnart_lag1+
                          #productstar_EP_lag1+      
                          productstar_Hbeach_lag1+
                         # productstar_secura_lag1+
                         # productstar_tfal_lag1+
                         Product_reviews_Hbeach_lag1+
                          Product_reviews_secura_lag1+
                         Product_reviews_tfal_lag1+
                          weekend	+ 
                          seasonal_sale +
                         ###4th round additions
                          mean_primeCSN_3p_lag1+
                         maxpricechange_HD +
                         maxpricechange_HDlag1 +
                         maxpricechange_sears +
                         maxpricechange_searslag1+
                         maxpricechange_wm  
                         #maxpricechange_wmlag1
                        ,family= gaussian(link ='identity') ,data=data_amzn_csnart ,method="REML",optimizer=c("outer","newton"), fit = TRUE)

summary(gam7_amzn_csnart)

#AMAZON PRESTO


data_amzn_presto = data.frame(cbind(Y_amzn_presto_lag1,Y_amzn_presto_lag2,Y_amzn_presto_lag3,   
                                    Y_clust4_secura_lag1,
                                    min_cumsalefreq_clust5_lag1,
                                    num_csnartclust3_lag1,
                                    meannumsellerate_clust3_lag1,
                                    Y_amzn_csnart_lag1,
                                    buyboxprice_csnart_lag1,
                                    mean_uniq_ASIN_clust1_lag1,
                                    maxpercsellerate_clust2_lag1,
                                    Y_amzn_tfal_lag1,
                                    ep_topbrandlag1	,
                                    presto_topbrandlag1	,
                                    hbeach_topbrandlag1	,
                                    csnart_topbrandlag1	,
                                    secura_topbrandlag1	,
                                    tfal_topbrandlag1	,
                                    ep_bottombrandlag1	,
                                    presto_bottombrandlag1	,
                                    hbeach_bottombrandlag1	,
                                    csnart_bottombrandlag1	,
                                    secura_bottombrandlag1	,
                                    tfal_bottombrandlag1	,
                                    decr10per_3p_tfal	,
                                    decr10per_amzn_hbeach	,
                                    decr10per_amzn_tfal	,
                                    decr10per_clust3_csnart	,
                                    decr20per_3p_sec	,
                                    decr20per_amzn_csnart	,
                                    decr20per_amzn_presto	,
                                    decr20per_clust1_csnart	,
                                    decr20per_clust5_tfal	,
                                    decr5per_3p_tfal	,
                                    decr5per_amzn_hbeach	,
                                    decr5per_amzn_tfal	,
                                    decr5per_clust3_csnart	,
                                    incr10per_3p_sec	,
                                    incr10per_amzn_csnart	,
                                    incr10per_amzn_presto	,
                                    incr10per_clust1_csnart	,
                                    incr10per_clust5_tfal	,
                                    incr20per_3p_tfal	,
                                    incr20per_amzn_hbeach	,
                                    incr20per_amzn_tfal	,
                                    incr20per_clust3_csnart	,
                                    incr5per_3p_sec	,
                                    incr5per_amzn_csnart	,
                                    incr5per_amzn_presto	,
                                    incr5per_clust1_csnart	,
                                    incr5per_clust5_tfal	,
                                    decr10per_3p_sec	,
                                    decr10per_amzn_csnart	,
                                    decr10per_amzn_presto	,
                                    decr10per_clust1_csnart	,
                                    decr10per_clust5_tfal	,
                                    decr20per_3p_tfal	,
                                    decr20per_amzn_hbeach	,
                                    decr20per_amzn_tfal	,
                                    decr20per_clust3_csnart	,
                                    decr5per_3p_sec	,
                                    decr5per_amzn_csnart	,
                                    decr5per_amzn_presto	,
                                    decr5per_clust1_csnart	,
                                    decr5per_clust5_tfal	,
                                    incr10per_3p_tfal	,
                                    incr10per_amzn_hbeach	,
                                    incr10per_amzn_tfal	,
                                    incr10per_clust3_csnart	,
                                    incr20per_3p_sec	,
                                    incr20per_amzn_csnart	,
                                    incr20per_amzn_presto	,
                                    incr20per_clust1_csnart	,
                                    incr20per_clust5_tfal	,
                                    incr5per_3p_tfal	,
                                    incr5per_amzn_hbeach	,
                                    incr5per_amzn_tfal	,
                                    incr5per_clust3_csnart	,
                                    weekend	,
                                    #additional variables on product reviews
                                    max_buyboxprice_presto_lag1,
                                    mean_buyboxprice_presto_lag1,
                                    max_Product_reviews_presto_lag1,
                                    Product_reviews_csnart_lag1,
                                    Product_reviews_Hbeach_lag1,
                                    Product_reviews_secura_lag1,
                                    Product_reviews_tfal_lag1,
                                    meanprodstar_presto_lag1,
                                    productstar_csnart_lag1,
                                    productstar_EP_lag1,      
                                    productstar_Hbeach_lag1,
                                    productstar_secura_lag1,
                                    productstar_tfal_lag1,
                                    #additional FEs after 2nd RnR
                                    mean_starratePR_3p_lag1,
                                    mean_primePR_3p_lag1,
                                    seasonal_sale,
                                    ###broken down external site price changes with lags
                                    maxpricechange_HD,
                                    maxpricechange_HDlag1,
                                    maxpricechange_HDlag2,
                                    maxpricechange_HDlag3,
                                    maxpricechange_sears,
                                    maxpricechange_searslag1,
                                    maxpricechange_searslag2,
                                    maxpricechange_searslag3,
                                    maxpricechange_wm,
                                    maxpricechange_wmlag1,
                                    maxpricechange_wmlag2,
                                    maxpricechange_wmlag3))


#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS
gam5_amzn_presto  = gam(Y_amzn_presto  ~ Y_amzn_presto_lag1 + 
                          min_cumsalefreq_clust5_lag1+
                         # meannumsellerate_clust3_lag1+
                          #Y_amzn_csnart_lag1 +
                        #  mean_uniq_ASIN_clust1_lag1+
                          mean_buyboxprice_presto_lag1+
                          #ep_topbrandlag1	+
                          #presto_topbrandlag1	+
                          #hbeach_topbrandlag1	+
                          #csnart_topbrandlag1	+
                         # secura_topbrandlag1	+
                          #tfal_topbrandlag1	+
                          #ep_bottombrandlag1	+
                          #presto_bottombrandlag1	+
                          #hbeach_bottombrandlag1	+
                          # csnart_bottombrandlag1	+
                          # secura_bottombrandlag1	+
                        #tfal_bottombrandlag1	+
                          # decr10per_3p_tfal	+
                          #decr10per_amzn_hbeach	+
                          #decr10per_amzn_tfal	+
                          #decr10per_clust3_csnart	+
                          #decr20per_amzn_csnart	+
                       #   decr20per_clust1_csnart	+
                          #decr5per_3p_tfal	+
                          #decr5per_amzn_hbeach	+
                          #decr5per_amzn_tfal	+
                          #decr5per_clust3_csnart	+
                          #incr10per_amzn_csnart	+
                          incr10per_amzn_presto	+
                          #incr10per_clust1_csnart	+
                          #incr20per_3p_tfal	+
                          #incr20per_amzn_hbeach	+
                         # incr20per_amzn_tfal	+
                          #incr5per_amzn_csnart	+
                        # incr5per_amzn_presto	+
                          #incr5per_clust1_csnart	+
                          #decr10per_amzn_csnart	+
                          #decr10per_amzn_presto	+
                         # decr10per_clust1_csnart	+
                          #decr20per_amzn_hbeach	+
                        #  decr20per_amzn_tfal	+
                          #decr5per_amzn_csnart	+
                          decr5per_amzn_presto	+
                          #decr5per_clust1_csnart	+
                          #incr10per_3p_tfal	+
                          # incr10per_amzn_hbeach	+
                         # incr10per_amzn_tfal	+
                          #incr20per_amzn_csnart	+
                          #incr20per_amzn_presto	+
                          #incr20per_clust1_csnart	+
                         # incr5per_3p_tfal	+
                        #  incr5per_amzn_hbeach	+
                         # incr5per_amzn_tfal	+
                          # incr5per_clust3_csnart	+
                        #additional variables on product reviews
                       #  max_Product_reviews_presto_lag1+
                        #  Product_reviews_csnart_lag1+
                        #  Product_reviews_Hbeach_lag1+
                         # Product_reviews_secura_lag1+
                          Product_reviews_tfal_lag1+
                            meanprodstar_presto_lag1+
                            #productstar_csnart_lag1+
                           # productstar_EP_lag1+      
                           # productstar_Hbeach_lag1+
                          #  productstar_secura_lag1+
                           # productstar_tfal_lag1+
                            weekend	+
                        #  mean_starratePR_3p_lag1 +
                        seasonal_sale+
                         ###4th round additions
                         maxpricechange_HD +
                         #maxpricechange_HDlag1+
                         #maxpricechange_HDlag2 +
                        # maxpricechange_HDlag3 +
                        # maxpricechange_sears +
                        # maxpricechange_searslag1+
                        # maxpricechange_searslag2+
                         maxpricechange_searslag3+
                         #maxpricechange_wm  +
                         maxpricechange_wmlag1 
                        # maxpricechange_wmlag2 + maxpricechange_wmlag3
                       ,family= gaussian(link ='identity') ,data=data_amzn_presto ,method="REML",optimizer=c("outer","newton"), fit = TRUE)

summary(gam5_amzn_presto)




#CLUSTER 1
#trainY = cbind(Y_amzn_tfal,Y_amzn_hbeach,Y_amzn_csnart,Y_amzn_presto,Y_clust1_csnart,Y_clust3_csnart,Y_clust4_secura,Y_clust5_tfal)

data_clust1 = data.frame(cbind(Y_clust1_csnart_lag1,
                               Y_clust1_csnart_lag2,
                               Y_clust1_csnart_lag3,
                               Y_clust4_secura_lag1,
                               min_cumsalefreq_clust5_lag1,
                               num_csnartclust3_lag1,
                               meannumsellerate_clust3_lag1,
                               Y_amzn_csnart_lag1,
                               buyboxprice_csnart_lag1,
                               mean_uniq_ASIN_clust1_lag1,
                               maxpercsellerate_clust2_lag1,
                               Y_amzn_tfal_lag1,
                               ep_topbrandlag1	,
                               presto_topbrandlag1	,
                               hbeach_topbrandlag1	,
                               csnart_topbrandlag1	,
                               secura_topbrandlag1	,
                               tfal_topbrandlag1	,
                               ep_bottombrandlag1	,
                               presto_bottombrandlag1	,
                               hbeach_bottombrandlag1	,
                               csnart_bottombrandlag1	,
                               secura_bottombrandlag1	,
                               tfal_bottombrandlag1	,
                               decr10per_3p_tfal	,
                               decr10per_amzn_hbeach	,
                               decr10per_amzn_tfal	,
                               decr10per_clust3_csnart	,
                               decr20per_3p_sec	,
                               decr20per_amzn_csnart	,
                               decr20per_amzn_presto	,
                               decr20per_clust1_csnart	,
                               decr20per_clust5_tfal	,
                               decr5per_3p_tfal	,
                               decr5per_amzn_hbeach	,
                               decr5per_amzn_tfal	,
                               decr5per_clust3_csnart	,
                               incr10per_3p_sec	,
                               incr10per_amzn_csnart	,
                               incr10per_amzn_presto	,
                               incr10per_clust1_csnart	,
                               incr10per_clust5_tfal	,
                               incr20per_3p_tfal	,
                               incr20per_amzn_hbeach	,
                               incr20per_amzn_tfal	,
                               incr20per_clust3_csnart	,
                               incr5per_3p_sec	,
                               incr5per_amzn_csnart	,
                               incr5per_amzn_presto	,
                               incr5per_clust1_csnart	,
                               incr5per_clust5_tfal	,
                               decr10per_3p_sec	,
                               decr10per_amzn_csnart	,
                               decr10per_amzn_presto	,
                               decr10per_clust1_csnart	,
                               decr10per_clust5_tfal	,
                               decr20per_3p_tfal	,
                               decr20per_amzn_hbeach	,
                               decr20per_amzn_tfal	,
                               decr20per_clust3_csnart	,
                               decr5per_3p_sec	,
                               decr5per_amzn_csnart	,
                               decr5per_amzn_presto	,
                               decr5per_clust1_csnart	,
                               decr5per_clust5_tfal	,
                               incr10per_3p_tfal	,
                               incr10per_amzn_hbeach	,
                               incr10per_amzn_tfal	,
                               incr10per_clust3_csnart	,
                               incr20per_3p_sec	,
                               incr20per_amzn_csnart	,
                               incr20per_amzn_presto	,
                               incr20per_clust1_csnart	,
                               incr20per_clust5_tfal	,
                               incr5per_3p_tfal	,
                               incr5per_amzn_hbeach	,
                               incr5per_amzn_tfal	,
                               incr5per_clust3_csnart	,
                               weekend	,
                               #additional variables on product reviews
                               max_Product_reviews_presto_lag1,
                               Product_reviews_csnart_lag1,
                               Product_reviews_Hbeach_lag1,
                               Product_reviews_secura_lag1,
                               Product_reviews_tfal_lag1,
                               meanprodstar_presto_lag1,
                               productstar_csnart_lag1,
                               productstar_EP_lag1,      
                               productstar_Hbeach_lag1,
                               productstar_secura_lag1,
                               productstar_tfal_lag1,
                               #additional FEs after 2nd RnR
                               mean_starrateCSN_3p_lag1,
                               mean_primeCSN_3p_lag1,
                               seasonal_sale,
                               ###broken down external site price changes with lags
                               maxpricechange_HD,
                               maxpricechange_HDlag1,
                               maxpricechange_HDlag2,
                               maxpricechange_HDlag3,
                               maxpricechange_sears,
                               maxpricechange_searslag1,
                               maxpricechange_searslag2,
                               maxpricechange_searslag3,
                               maxpricechange_wm,
                               maxpricechange_wmlag1,
                               maxpricechange_wmlag2,
                               maxpricechange_wmlag3))

#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS
gam9_clust1 = gam(Y_clust1_csnart ~  
                    #Y_clust4_secura_lag1 +
                   # min_cumsalefreq_clust5_lag1+
                    num_csnartclust3_lag1+
                  #  meannumsellerate_clust3_lag1+
                   # Y_amzn_csnart_lag1+
                    buyboxprice_csnart_lag1+
                    mean_uniq_ASIN_clust1_lag1+
                   # maxpercsellerate_clust2_lag1+
                   # Y_amzn_tfal_lag1+
                    #ep_topbrandlag1	+
                    #presto_topbrandlag1	+
                    #hbeach_topbrandlag1	+
                    #csnart_topbrandlag1	+
                   # secura_topbrandlag1	+
                    #tfal_topbrandlag1	+
                    #ep_bottombrandlag1	+
                    #presto_bottombrandlag1	+
                    #hbeach_bottombrandlag1	+
                    # csnart_bottombrandlag1	+
                    # secura_bottombrandlag1	+
                    #tfal_bottombrandlag1	+
                  # decr10per_3p_tfal	+
                  #decr10per_amzn_hbeach	+
                    #decr10per_amzn_tfal	+
                    #decr10per_clust3_csnart	+
                  #  decr20per_amzn_csnart	+
                  #  decr20per_clust1_csnart	+
                    #decr5per_3p_tfal	+
                    #decr5per_amzn_hbeach	+
                    #decr5per_amzn_tfal	+
                    #decr5per_clust3_csnart	+
                    #incr10per_amzn_csnart	+
                    #incr10per_amzn_presto	+
                    #incr10per_clust1_csnart	+
                    #incr20per_3p_tfal	+
                    #incr20per_amzn_hbeach	+
                    # incr20per_amzn_tfal	+
                    #incr5per_amzn_csnart	+
                    # incr5per_amzn_presto	+
                   # incr5per_clust1_csnart	+
                    #decr10per_amzn_csnart	+
                    #decr10per_amzn_presto	+
                    #decr10per_clust1_csnart	+
                   # decr20per_amzn_hbeach	+
                    #  decr20per_amzn_tfal	+
                    decr5per_amzn_csnart	+
                    #decr5per_amzn_presto	+
                    #decr5per_clust1_csnart	+
                    #incr10per_3p_tfal	+
                    # incr10per_amzn_hbeach	+
                    # incr10per_amzn_tfal	+
                    #incr20per_amzn_csnart	+
                    #incr20per_amzn_presto	+
                    incr20per_clust1_csnart	+
                    # incr5per_3p_tfal	+
                     # incr5per_amzn_hbeach	+
                    # incr5per_amzn_tfal	+
                    # incr5per_clust3_csnart	+
                    weekend+
                    mean_starrateCSN_3p_lag1+
                    #additional variables on product reviews
                    #max_Product_reviews_presto_lag1+
                   # Product_reviews_csnart_lag1+
                    #Product_reviews_Hbeach_lag1+
                    Product_reviews_secura_lag1+
                   # Product_reviews_tfal_lag1+
                   # meanprodstar_presto_lag1+
                    productstar_csnart_lag1+
                   # productstar_EP_lag1+      
                    productstar_Hbeach_lag1+
                    productstar_secura_lag1+
                    #productstar_tfal_lag1+
                    seasonal_sale +
                    ###4th round additions
                    #maxpricechange_HD +
                    maxpricechange_HDlag1+
                    #maxpricechange_HDlag2 +
                    #maxpricechange_HDlag3 +
                    #maxpricechange_sears +
                    maxpricechange_searslag1+
                    #maxpricechange_searslag2+
                    #maxpricechange_searslag3+
                    #maxpricechange_wm  +
                    maxpricechange_wmlag1 
                   # maxpricechange_wmlag2 + maxpricechange_wmlag3
                  ,family= gaussian(link ='identity') ,data=data_clust1,method="REML",optimizer=c("outer","newton"), fit = TRUE)

summary(gam9_clust1)


#CLUSTER 3

data_clust3 = data.frame(cbind(Y_clust3_csnart_lag1, 
                               Y_clust3_csnart_lag2,
                               Y_clust3_csnart_lag3,
                               Y_clust4_secura_lag1,
                               min_cumsalefreq_clust5_lag1,
                               num_csnartclust3_lag1,
                               meannumsellerate_clust3_lag1,
                               Y_amzn_csnart_lag1,
                               buyboxprice_csnart_lag1,
                               mean_uniq_ASIN_clust1_lag1,
                               maxpercsellerate_clust2_lag1,
                               Y_amzn_tfal_lag1,
                               ep_topbrandlag1	,
                               presto_topbrandlag1	,
                               hbeach_topbrandlag1	,
                               csnart_topbrandlag1	,
                               secura_topbrandlag1	,
                               tfal_topbrandlag1	,
                               ep_bottombrandlag1	,
                               presto_bottombrandlag1	,
                               hbeach_bottombrandlag1	,
                               csnart_bottombrandlag1	,
                               secura_bottombrandlag1	,
                               tfal_bottombrandlag1	,
                               decr10per_3p_tfal	,
                               decr10per_amzn_hbeach	,
                               decr10per_amzn_tfal	,
                               decr10per_clust3_csnart	,
                               decr20per_3p_sec	,
                               decr20per_amzn_csnart	,
                               decr20per_amzn_presto	,
                               decr20per_clust1_csnart	,
                               decr20per_clust5_tfal	,
                               decr5per_3p_tfal	,
                               decr5per_amzn_hbeach	,
                               decr5per_amzn_tfal	,
                               decr5per_clust3_csnart	,
                               incr10per_3p_sec	,
                               incr10per_amzn_csnart	,
                               incr10per_amzn_presto	,
                               incr10per_clust1_csnart	,
                               incr10per_clust5_tfal	,
                               incr20per_3p_tfal	,
                               incr20per_amzn_hbeach	,
                               incr20per_amzn_tfal	,
                               incr20per_clust3_csnart	,
                               incr5per_3p_sec	,
                               incr5per_amzn_csnart	,
                               incr5per_amzn_presto	,
                               incr5per_clust1_csnart	,
                               incr5per_clust5_tfal	,
                               decr10per_3p_sec	,
                               decr10per_amzn_csnart	,
                               decr10per_amzn_presto	,
                               decr10per_clust1_csnart	,
                               decr10per_clust5_tfal	,
                               decr20per_3p_tfal	,
                               decr20per_amzn_hbeach	,
                               decr20per_amzn_tfal	,
                               decr20per_clust3_csnart	,
                               decr5per_3p_sec	,
                               decr5per_amzn_csnart	,
                               decr5per_amzn_presto	,
                               decr5per_clust1_csnart	,
                               decr5per_clust5_tfal	,
                               incr10per_3p_tfal	,
                               incr10per_amzn_hbeach	,
                               incr10per_amzn_tfal	,
                               incr10per_clust3_csnart	,
                               incr20per_3p_sec	,
                               incr20per_amzn_csnart	,
                               incr20per_amzn_presto	,
                               incr20per_clust1_csnart	,
                               incr20per_clust5_tfal	,
                               incr5per_3p_tfal	,
                               incr5per_amzn_hbeach	,
                               incr5per_amzn_tfal	,
                               incr5per_clust3_csnart	,
                               weekend	,
                               #additional variables on product reviews
                               max_Product_reviews_presto_lag1,
                               Product_reviews_csnart_lag1,
                               Product_reviews_Hbeach_lag1,
                               Product_reviews_secura_lag1,
                               Product_reviews_tfal_lag1,
                               meanprodstar_presto_lag1,
                               productstar_csnart_lag1,
                               productstar_EP_lag1,      
                               productstar_Hbeach_lag1,
                               productstar_secura_lag1,
                               productstar_tfal_lag1,
                               #additional FEs after 2nd RnR
                               mean_starrateCSN_3p_lag1,
                               mean_primeCSN_3p_lag1,
                               seasonal_sale,
                               ###broken down external site price changes with lags
                               maxpricechange_HD,
                               maxpricechange_HDlag1,
                               maxpricechange_HDlag2,
                               maxpricechange_HDlag3,
                               maxpricechange_sears,
                               maxpricechange_searslag1,
                               maxpricechange_searslag2,
                               maxpricechange_searslag3,
                               maxpricechange_wm,
                               maxpricechange_wmlag1,
                               maxpricechange_wmlag2,
                               maxpricechange_wmlag3))

#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS
gam9_clust3 = gam(Y_clust3_csnart ~  
                    #Y_clust3_csnart_lag1	+
                    #Y_clust3_csnart_lag2	+
                    #Y_clust3_csnart_lag3	+
                    min_cumsalefreq_clust5_lag1+
                    buyboxprice_csnart_lag1+
                    num_csnartclust3_lag1+
                    meannumsellerate_clust3_lag1+
                    incr10per_clust3_csnart +
                    #Y_amzn_csnart_lag1+
                   # maxpercsellerate_clust2_lag1+
                    #Y_amzn_tfal_lag1+
                    #ep_topbrandlag1	+
                    #presto_topbrandlag1	+
                    #hbeach_topbrandlag1	+
                   # csnart_topbrandlag1	+
                    # secura_topbrandlag1	+
                    #tfal_topbrandlag1	+
                    #ep_bottombrandlag1	+
                    #presto_bottombrandlag1	+
                    #hbeach_bottombrandlag1	+
                    # csnart_bottombrandlag1	+
                    # secura_bottombrandlag1	+
                    #tfal_bottombrandlag1	+
                  # decr10per_3p_tfal	+
                  #decr10per_amzn_hbeach	+
                    #decr10per_amzn_tfal	+
                    #decr10per_clust3_csnart	+
                   # decr20per_amzn_csnart	+
                    decr20per_clust1_csnart	+
                    #decr5per_3p_tfal	+
                  #  decr5per_amzn_hbeach	+
                    #decr5per_amzn_tfal	+
                    #decr5per_clust3_csnart	+
                    #incr10per_amzn_csnart	+
                    #incr10per_amzn_presto	+
                   # incr10per_clust1_csnart	+
                    #incr20per_3p_tfal	+
                    #incr20per_amzn_hbeach	+
                    # incr20per_amzn_tfal	+
                    #incr5per_amzn_csnart	+
                    # incr5per_amzn_presto	+
                    incr5per_clust1_csnart	+
                    #decr10per_amzn_csnart	+
                    #decr10per_amzn_presto	+
                    #decr10per_clust1_csnart	+
                   # decr20per_amzn_hbeach	+
                    #  decr20per_amzn_tfal	+
                  #  decr5per_amzn_csnart	+
                    #decr5per_amzn_presto	+
                   # decr5per_clust1_csnart	+
                    #incr10per_3p_tfal	+
                    # incr10per_amzn_hbeach	+
                    # incr10per_amzn_tfal	+
                    #incr20per_amzn_csnart	+
                    #incr20per_amzn_presto	+
                    #incr20per_clust1_csnart	+
                    # incr5per_3p_tfal	+
                   # incr5per_amzn_hbeach	+
                    # incr5per_amzn_tfal	+
                     #incr5per_clust3_csnart	+
                     weekend+
                    #additional variables on product reviews
                    #max_Product_reviews_presto_lag1+
                    Product_reviews_csnart_lag1+
                    #Product_reviews_Hbeach_lag1+
                    Product_reviews_secura_lag1+
                    #Product_reviews_tfal_lag1+
                    meanprodstar_presto_lag1+
                    productstar_csnart_lag1+
                    productstar_EP_lag1+      
                    #productstar_Hbeach_lag1+
                    productstar_secura_lag1+
                   # productstar_tfal_lag1+
                    #additional variables from RnR round 2
                   # mean_starrateCSN_3p_lag1 +
                   # mean_primeCSN_3p_lag1 +
                    seasonal_sale +
                    ###4th round additions
                   # maxpricechange_HD +
                    maxpricechange_HDlag1+
                  #  maxpricechange_HDlag2 +
                    #maxpricechange_HDlag3 +
                    #maxpricechange_sears +
                    maxpricechange_searslag2+
                    #maxpricechange_searslag2+
                   # maxpricechange_searslag3+
                   # maxpricechange_wm  +
                    maxpricechange_wmlag2
                  # maxpricechange_wmlag2 + maxpricechange_wmlag3
                    ,family= gaussian(link ='identity') ,data=data_clust3,method="REML",optimizer=c("outer","newton"), fit = TRUE)


summary(gam9_clust3)

#CLUSTER 4

data_clust4 = data.frame(cbind(Y_clust4_secura_lag1,
                               Y_clust4_secura_lag2,
                               Y_clust4_secura_lag3,
                               min_cumsalefreq_clust5_lag1,
                               num_csnartclust3_lag1,
                               meannumsellerate_clust3_lag1,
                               Y_amzn_csnart_lag1,
                               buyboxprice_csnart_lag1,
                               buyboxprice_secura_lag1,
                               mean_uniq_ASIN_clust1_lag1,
                               maxpercsellerate_clust2_lag1,
                               Y_amzn_tfal_lag1,
                               ep_topbrandlag1	,
                               presto_topbrandlag1	,
                               hbeach_topbrandlag1	,
                               csnart_topbrandlag1	,
                               secura_topbrandlag1	,
                               tfal_topbrandlag1	,
                               ep_bottombrandlag1	,
                               presto_bottombrandlag1	,
                               hbeach_bottombrandlag1	,
                               csnart_bottombrandlag1	,
                               secura_bottombrandlag1	,
                               tfal_bottombrandlag1	,
                               decr10per_3p_tfal	,
                               decr10per_amzn_hbeach	,
                               decr10per_amzn_tfal	,
                               decr10per_clust3_csnart	,
                               decr20per_3p_sec	,
                               decr20per_amzn_csnart	,
                               decr20per_amzn_presto	,
                               decr20per_clust1_csnart	,
                               decr20per_clust5_tfal	,
                               decr5per_3p_tfal	,
                               decr5per_amzn_hbeach	,
                               decr5per_amzn_tfal	,
                               decr5per_clust3_csnart	,
                               incr10per_3p_sec	,
                               incr10per_amzn_csnart	,
                               incr10per_amzn_presto	,
                               incr10per_clust1_csnart	,
                               incr10per_clust5_tfal	,
                               incr20per_3p_tfal	,
                               incr20per_amzn_hbeach	,
                               incr20per_amzn_tfal	,
                               incr20per_clust3_csnart	,
                               incr5per_3p_sec	,
                               incr5per_amzn_csnart	,
                               incr5per_amzn_presto	,
                               incr5per_clust1_csnart	,
                               incr5per_clust5_tfal	,
                               decr10per_3p_sec	,
                               decr10per_amzn_csnart	,
                               decr10per_amzn_presto	,
                               decr10per_clust1_csnart	,
                               decr10per_clust5_tfal	,
                               decr20per_3p_tfal	,
                               decr20per_amzn_hbeach	,
                               decr20per_amzn_tfal	,
                               decr20per_clust3_csnart	,
                               decr5per_3p_sec	,
                               decr5per_amzn_csnart	,
                               decr5per_amzn_presto	,
                               decr5per_clust1_csnart	,
                               decr5per_clust5_tfal	,
                               incr10per_3p_tfal	,
                               incr10per_amzn_hbeach	,
                               incr10per_amzn_tfal	,
                               incr10per_clust3_csnart	,
                               incr20per_3p_sec	,
                               incr20per_amzn_csnart	,
                               incr20per_amzn_presto	,
                               incr20per_clust1_csnart	,
                               incr20per_clust5_tfal	,
                               incr5per_3p_tfal	,
                               incr5per_amzn_hbeach	,
                               incr5per_amzn_tfal	,
                               incr5per_clust3_csnart	,
                               weekend	,
                               #additional variables on product reviews
                               max_Product_reviews_presto_lag1,
                               Product_reviews_csnart_lag1,
                               Product_reviews_Hbeach_lag1,
                               Product_reviews_secura_lag1,
                               Product_reviews_tfal_lag1,
                               meanprodstar_presto_lag1,
                               productstar_csnart_lag1,
                               productstar_EP_lag1,      
                               productstar_Hbeach_lag1,
                               productstar_secura_lag1,
                               productstar_tfal_lag1,
                               #additional FEs after 2nd RnR
                               mean_starrateSEC_3p_lag1,
                               mean_primeSEC_3p_lag1,
                               seasonal_sale,
                               ###broken down external site price changes with lags
                               maxpricechange_HD,
                               maxpricechange_HDlag1,
                               maxpricechange_HDlag2,
                               maxpricechange_HDlag3,
                               maxpricechange_sears,
                               maxpricechange_searslag1,
                               maxpricechange_searslag2,
                               maxpricechange_searslag3,
                               maxpricechange_wm,
                               maxpricechange_wmlag1,
                               maxpricechange_wmlag2,
                               maxpricechange_wmlag3))

#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS
gam9_clust4 = gam(Y_clust4_secura ~  
                    num_csnartclust3_lag1+
                    meannumsellerate_clust3_lag1+
                    Y_amzn_csnart_lag1 +
                    buyboxprice_secura_lag1+
                   # buyboxprice_csnart_lag1+
                    #maxpercsellerate_clust2_lag1+
                  #  Y_amzn_tfal_lag1+
                    #ep_topbrandlag1	+
                    #presto_topbrandlag1	+
                    #hbeach_topbrandlag1	+
                    #csnart_topbrandlag1	+
                    # secura_topbrandlag1	+
                    #tfal_topbrandlag1	+
                    #ep_bottombrandlag1	+
                    #presto_bottombrandlag1	+
                    #hbeach_bottombrandlag1	+
                    #csnart_bottombrandlag1	+
                    # secura_bottombrandlag1	+
                    #tfal_bottombrandlag1	+
                  # decr10per_3p_tfal	+
                #  decr10per_amzn_hbeach	+
                    #decr10per_amzn_tfal	+
                   # decr10per_clust3_csnart	+
                    #decr20per_amzn_csnart	+
                   # decr20per_clust1_csnart	+
                    #decr5per_3p_tfal	+
                    #decr5per_amzn_hbeach	+
                    #decr5per_amzn_tfal	+
                    #decr5per_clust3_csnart	+
                   # incr10per_amzn_csnart	+
                    #incr10per_amzn_presto	+
                   #  incr10per_clust1_csnart	+
                    #incr20per_3p_tfal	+
                    #incr20per_amzn_hbeach	+
                    # incr20per_amzn_tfal	+
                    #incr5per_amzn_csnart	+
                    # incr5per_amzn_presto	+
                   # incr5per_clust1_csnart	+
                    #decr10per_amzn_csnart	+
                    #decr10per_amzn_presto	+
                    #decr10per_clust1_csnart	+
                    # decr20per_amzn_hbeach	+
                    #  decr20per_amzn_tfal	+
                   #  decr5per_amzn_csnart	+
                    #decr5per_amzn_presto	+
                    #decr5per_clust1_csnart	+
                   # incr10per_3p_tfal	+
                    # incr10per_amzn_hbeach	+
                     incr10per_amzn_tfal	+
                    #incr20per_amzn_csnart	+
                    #incr20per_amzn_presto	+
                    #incr20per_clust1_csnart	+
                    # incr5per_3p_tfal	+
                    # incr5per_amzn_hbeach	+
                      incr5per_amzn_tfal	+
                    # incr5per_clust3_csnart	+
                     weekend +
                    max_Product_reviews_presto_lag1+
                 #   Product_reviews_csnart_lag1+
                    #Product_reviews_Hbeach_lag1+
                    Product_reviews_secura_lag1+
                    Product_reviews_tfal_lag1+
                 # meanprodstar_presto_lag1+
                #  productstar_csnart_lag1+
                 # productstar_EP_lag1+      
                #  productstar_Hbeach_lag1+
                  productstar_secura_lag1+
                  productstar_tfal_lag1+
                  #additional variables from JM RnR Round 2
               #   mean_starrateSEC_3p_lag1 +
                seasonal_sale +
                  ###4th round additions
                  # maxpricechange_HD +
                 # maxpricechange_HDlag1+
                 # maxpricechange_HDlag2 +
                  maxpricechange_HDlag3 +
                  #maxpricechange_sears +
                  #maxpricechange_searslag1+
                  #maxpricechange_searslag2+
                   maxpricechange_searslag3+
                  # maxpricechange_wm  +
                  #maxpricechange_wmlag1 +
                  # maxpricechange_wmlag2 +
                   maxpricechange_wmlag3 
                    #season
                  ,family= gaussian(link ='identity') ,data=data_clust4,method="REML",optimizer=c("outer","newton"), fit = TRUE)

summary(gam9_clust4)


#CLUSTER 5

data_clust5 = data.frame(cbind(Y_clust5_tfal_lag1,
                               Y_clust5_tfal_lag2,
                               Y_clust5_tfal_lag3,
                               Y_clust4_secura_lag1,
                               min_cumsalefreq_clust5_lag1,
                               num_csnartclust3_lag1,
                               meannumsellerate_clust3_lag1,
                               Y_amzn_csnart_lag1,
                               buyboxprice_csnart_lag1,
                               mean_uniq_ASIN_clust1_lag1,
                               maxpercsellerate_clust2_lag1,
                               Y_amzn_tfal_lag1,
                               ep_topbrandlag1	,
                               presto_topbrandlag1	,
                               hbeach_topbrandlag1	,
                               csnart_topbrandlag1	,
                               secura_topbrandlag1	,
                               tfal_topbrandlag1	,
                               ep_bottombrandlag1	,
                               presto_bottombrandlag1	,
                               hbeach_bottombrandlag1	,
                               csnart_bottombrandlag1	,
                               secura_bottombrandlag1	,
                               tfal_bottombrandlag1	,
                               decr10per_3p_tfal	,
                               decr10per_amzn_hbeach	,
                               decr10per_amzn_tfal	,
                               decr10per_clust3_csnart	,
                               decr20per_3p_sec	,
                               decr20per_amzn_csnart	,
                               decr20per_amzn_presto	,
                               decr20per_clust1_csnart	,
                               decr20per_clust5_tfal	,
                               decr5per_3p_tfal	,
                               decr5per_amzn_hbeach	,
                               decr5per_amzn_tfal	,
                               decr5per_clust3_csnart	,
                               incr10per_3p_sec	,
                               incr10per_amzn_csnart	,
                               incr10per_amzn_presto	,
                               incr10per_clust1_csnart	,
                               incr10per_clust5_tfal	,
                               incr20per_3p_tfal	,
                               incr20per_amzn_hbeach	,
                               incr20per_amzn_tfal	,
                               incr20per_clust3_csnart	,
                               incr5per_3p_sec	,
                               incr5per_amzn_csnart	,
                               incr5per_amzn_presto	,
                               incr5per_clust1_csnart	,
                               incr5per_clust5_tfal	,
                               decr10per_3p_sec	,
                               decr10per_amzn_csnart	,
                               decr10per_amzn_presto	,
                               decr10per_clust1_csnart	,
                               decr10per_clust5_tfal	,
                               decr20per_3p_tfal	,
                               decr20per_amzn_hbeach	,
                               decr20per_amzn_tfal	,
                               decr20per_clust3_csnart	,
                               decr5per_3p_sec	,
                               decr5per_amzn_csnart	,
                               decr5per_amzn_presto	,
                               decr5per_clust1_csnart	,
                               decr5per_clust5_tfal	,
                               incr10per_3p_tfal	,
                               incr10per_amzn_hbeach	,
                               incr10per_amzn_tfal	,
                               incr10per_clust3_csnart	,
                               incr20per_3p_sec	,
                               incr20per_amzn_csnart	,
                               incr20per_amzn_presto	,
                               incr20per_clust1_csnart	,
                               incr20per_clust5_tfal	,
                               incr5per_3p_tfal	,
                               incr5per_amzn_hbeach	,
                               incr5per_amzn_tfal	,
                               incr5per_clust3_csnart	,
                               weekend	,
                               #additional variables on product reviews
                               max_Product_reviews_presto_lag1,
                               Product_reviews_csnart_lag1,
                               Product_reviews_Hbeach_lag1,
                               Product_reviews_secura_lag1,
                               Product_reviews_tfal_lag1,
                               meanprodstar_presto_lag1,
                               productstar_csnart_lag1,
                               productstar_EP_lag1,      
                               productstar_Hbeach_lag1,
                               productstar_secura_lag1,
                               productstar_tfal_lag1,
                               #additional FEs after 2nd RnR
                               mean_starrateCSN_3p_lag1,
                               mean_primeCSN_3p_lag1,
                               seasonal_sale,
                               ###broken down external site price changes with lags
                               maxpricechange_HD,
                               maxpricechange_HDlag1,
                               maxpricechange_HDlag2,
                               maxpricechange_HDlag3,
                               maxpricechange_sears,
                               maxpricechange_searslag1,
                               maxpricechange_searslag2,
                               maxpricechange_searslag3,
                               maxpricechange_wm,
                               maxpricechange_wmlag1,
                               maxpricechange_wmlag2,
                               maxpricechange_wmlag3))


#VARIANCE CHECK
variance_X = rep(0,ncol(data_clust5))

for (m in (1:ncol(data_clust5)))
{ variance_X[m] = var(data_clust5[,m]) }

nonzero_var = ncol(data_clust5)- sum((variance_X == 0)[]*1) #number of variables with non-zero variance

#Retaining variables with non-zero variance

trainX_retained = matrix(0,nrow(data_clust5),nonzero_var)

trainX_retained = data_clust5[,(variance_X != 0)]

#VARIANCE CHECK
variance_Xret = rep(0,ncol(trainX_retained))
for (m in (1:ncol(trainX_retained)))
{ variance_Xret[m] = var(trainX_retained[,m]) }

#TREATING INDICATOR VARIABLES 
summary(trainX_retained)

data_clust5 = trainX_retained

#ADDING FE & IE FOR SUBSTANTIVE INSIGHTS

gam5_clust5 = gam(Y_clust5_tfal ~  Y_clust5_tfal_lag1 +
                    #Y_clust4_secura_lag1+
                    min_cumsalefreq_clust5_lag1+
                 #   num_csnartclust3_lag1+
                   #Y_amzn_csnart_lag1 +
                    buyboxprice_tfal_lag1+
                    #mean_uniq_ASIN_clust1_lag1+
                   # Y_amzn_tfal_lag1+
                    #ep_topbrandlag1	+
                    #presto_topbrandlag1	+
                    #hbeach_topbrandlag1	+
                    #csnart_topbrandlag1	+
                    # secura_topbrandlag1	+
                    #tfal_topbrandlag1	+
                    #ep_bottombrandlag1	+
                    #presto_bottombrandlag1	+
                    #hbeach_bottombrandlag1	+
                    #csnart_bottombrandlag1	+
                    # secura_bottombrandlag1	+
                  #tfal_bottombrandlag1	+
                 # decr10per_3p_tfal	+
                   # decr10per_amzn_hbeach	+
                  #  decr10per_amzn_tfal	+
                   # decr10per_clust3_csnart	+
                   # decr20per_amzn_csnart	+
                    #decr20per_clust1_csnart	+
                   # decr5per_3p_tfal	+
                    #decr5per_amzn_hbeach	+
                   # decr5per_amzn_tfal	+
                   decr5per_amzn_tfal	+
                   # incr10per_amzn_csnart	+
                    #incr10per_amzn_presto	+
                   # incr10per_clust1_csnart	+
                    #incr20per_3p_tfal	+
                    #incr20per_amzn_hbeach	+
                    #incr20per_amzn_tfal	+
                    #incr5per_amzn_csnart	+
                    #incr5per_amzn_presto	+
                    #incr5per_clust1_csnart	+
                    #decr10per_amzn_csnart	+
                    #decr10per_amzn_presto	+
                    #decr10per_clust1_csnart	+
                   # decr20per_amzn_hbeach	+
                    #decr20per_amzn_tfal	+
                    #decr5per_amzn_csnart	+
                    #decr5per_amzn_presto	+
                    decr5per_clust1_csnart	+
                    #incr10per_3p_tfal	+
                    #incr10per_amzn_hbeach	+
                    #incr10per_amzn_tfal	+
                    #incr20per_amzn_csnart	+
                    #incr20per_amzn_presto	+
                    #incr20per_clust1_csnart	+
                    #incr5per_3p_tfal	+
                    #incr5per_amzn_hbeach	+
                   # incr5per_amzn_tfal	+
                    #incr5per_clust3_csnart	+
                    weekend +
                  #  max_Product_reviews_presto_lag1+
                   # Product_reviews_csnart_lag1+
                  #  Product_reviews_Hbeach_lag1+
                  #  Product_reviews_secura_lag1+
                    Product_reviews_tfal_lag1+
                     # meanprodstar_presto_lag1+
                    #  productstar_csnart_lag1+
                    #  productstar_EP_lag1+      
                     # productstar_Hbeach_lag1+
                      #productstar_secura_lag1+
                      productstar_tfal_lag1+
                     # season
                   #additional variables from RnR round 2
                   #mean_starrateCSN_3p_lag1+
                   #mean_primeCSN_3p_lag1+
                   seasonal_sale +
                   ###4th round additions
                    #maxpricechange_HD +
                   maxpricechange_HDlag1+
                    #maxpricechange_HDlag2 +
                   #maxpricechange_HDlag3 +
                   #maxpricechange_sears +
                   #maxpricechange_searslag1+
                   maxpricechange_searslag2+
                   #maxpricechange_searslag3+
                    #maxpricechange_wm  +
                   #maxpricechange_wmlag1 +
                    #maxpricechange_wmlag2 +
                   maxpricechange_wmlag3
                  ,family= gaussian(link ='identity') ,data=data_clust5,method="REML",optimizer=c("outer","newton"), fit = TRUE)

summary(gam5_clust5)


